package web.play.rss.util;

import java.io.File;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Hashtable;

import web.play.rss.util.dao.SpiderDAO;

public class GetWeb {
	private int webDepth = 2;

	private int intThreadNum = 10;

	private String strHomePage = "";

	private String myDomain;

	private String fPath = "web";

	private ArrayList<String> arrUrls = new ArrayList<String>();

	private ArrayList<String> arrUrl = new ArrayList<String>();

	private Hashtable<String, Integer> allUrls = new Hashtable<String, Integer>();

	private Hashtable<String, Integer> deepUrls = new Hashtable<String, Integer>();

	private int intWebIndex = 0;

	private String charset = "GBK";

	private String report = "";

	private long startTime;

	private int webSuccessed = 0;

	private int webFailed = 0;

	private String searchKey = "";

	private List searchedUrl = new ArrayList();

	public List searchedKey = new ArrayList();

	private boolean searchCurrent = false;

	private boolean saveUrl = true;

	private boolean saveSearchKey = true;

	private boolean debug = false;

	private void setDebug(boolean _d) {
		this.debug = _d;
	}

	public void setSaveSearchKey(boolean saveSearchKey) {
		this.saveSearchKey = saveSearchKey;
	}

	public void setSaveUrl(boolean saveUrl) {
		this.saveUrl = saveUrl;
	}

	public void setSearchCurrent(boolean searchCurrent) {
		this.searchCurrent = searchCurrent;
	}

	public String getSearchKey() {
		return searchKey;
	}

	public List getSearchedKey() {
		return searchedKey;
	}

	public List getSearchedUrl() {
		return searchedUrl;
	}

	public GetWeb(String s, String key) {
		this.strHomePage = s;
		this.searchKey = key;
	}

	public String getHomePage() {
		return this.strHomePage;
	}

	public GetWeb(String s, int i) {
		this.strHomePage = s;
		this.webDepth = i;
	}

	public synchronized void addWebSuccessed() {
		webSuccessed++;
	}

	public synchronized void addWebFailed() {
		webFailed++;
	}

	public synchronized void addReport(String s) {
		try {
			report += s;
			PrintWriter pwReport = new PrintWriter(new FileOutputStream(
					"report.txt"));
			pwReport.println(report);
			pwReport.close();
		} catch (Exception e) {
			System.out.println("aa");
		}
	}

	public synchronized String getAUrl() {
		String tmpAUrl = arrUrls.get(0);
		arrUrls.remove(0);
		return tmpAUrl;
	}

	public synchronized String getUrl() {
		String tmpUrl = arrUrl.get(0);
		arrUrl.remove(0);
		return tmpUrl;
	}

	public synchronized Integer getIntWebIndex() {
		intWebIndex++;
		return intWebIndex;
	}

	public void getWebByHomePage() {
		System.out.println("start ... ...");

		if (this.searchCurrent) {
			searchKey(getContent(this.getHomePage()), this.getSearchKey());
		}
		startTime = System.currentTimeMillis();
		this.myDomain = getDomain();
		if (myDomain == null) {
			System.out.println("Wrong input!");
			// System.exit(1);
			return;
		}
		// System.out.println("Homepage = " + strHomePage);addReport("Homepage =
		// " + strHomePage + "!\n");System.out.println("Domain = " +
		// myDomain);addReport("Domain = " + myDomain + "!\n");
		arrUrls.add(strHomePage);
		arrUrl.add(strHomePage);
		allUrls.put(strHomePage, 0);
		deepUrls.put(strHomePage, 1);
		// File fDir = new File(fPath);
		// if (!fDir.exists()) {
		// fDir.mkdir();
		// }
		// System.out.println("Start!");this.addReport("Start!\n");
		String tmp = getAUrl();
		this.getWebByUrl(tmp, charset, allUrls.get(tmp) + "");
		int i = 0;
		for (i = 0; i < intThreadNum; i++) {
			new Thread(new Processer(this)).start();
		}
		while (true) {
			if (arrUrls.isEmpty() && Thread.activeCount() == 1) {
				long finishTime = System.currentTimeMillis();
				long costTime = finishTime - startTime;
				System.out.println("\n\n\n\n\nFinished!");
				System.out.println("Start time = " + startTime + "   "
						+ "Finish time = " + finishTime + "   "
						+ "Cost time = " + costTime + "ms");
				// addReport("Start time = " + startTime + " "
				// + "Finish time = " + finishTime + " "
				// + "Cost time = " + costTime + "ms" + "\n");
				System.out.println("Total url number = "
						+ (webSuccessed + webFailed) + "   Successed: "
						+ webSuccessed + "   Failed: " + webFailed);
				// addReport("Total url number = " + (webSuccessed + webFailed)
				// + " Successed: " + webSuccessed + " Failed: "
				// + webFailed + "\n");

				String strIndex = "";
				String tmpUrl = "";
				while (!arrUrl.isEmpty()) {
					tmpUrl = getUrl();
					strIndex += "Web depth:" + deepUrls.get(tmpUrl)
							+ "   Filepath: " + fPath + "/web"
							+ allUrls.get(tmpUrl) + ".htm" + "   url:" + tmpUrl
							+ "\n\n";
				}
				// System.out.println(strIndex);
				try {
					PrintWriter pwIndex = new PrintWriter(new FileOutputStream(
							"fileindex.txt"));
					pwIndex.println(strIndex);
					pwIndex.close();
				} catch (Exception e) {
					System.out.println("bb");
				}
				break;
			}
		}
	}

	private String getContent(String strUrl) {
		try {
			URL pageUrl = new URL(strUrl);
			// Open connection to URL for reading.
			BufferedReader reader = new BufferedReader(new InputStreamReader(
					pageUrl.openStream()));

			// Read page into buffer.
			String line;
			StringBuffer pageBuffer = new StringBuffer();
			while ((line = reader.readLine()) != null) {
				pageBuffer.append(line);
				if (this.debug) {
					System.out.println(line);
				}
			}
			return pageBuffer.toString();
		} catch (Exception e) {
		}
		return null;
	}

	public void getWebByUrl(String strUrl, String charset, String fileIndex) {
		try {

			// if(charset==null||"".equals(charset))charset="utf-8";
			// System.out.println("Getting web by url: " + strUrl);
			// addReport("Getting web by url: " + strUrl + "\n");
			URL url = new URL(strUrl);
			URLConnection conn = url.openConnection();
			conn.setDoOutput(true);
			InputStream is = null;
			is = url.openStream();

			// String filePath = fPath + "/web" + fileIndex + ".htm";
			// PrintWriter pw = null;
			// FileOutputStream fos = new FileOutputStream(filePath);
			// OutputStreamWriter writer = new OutputStreamWriter(fos);
			// pw = new PrintWriter(writer);
			BufferedReader bReader = new BufferedReader(new InputStreamReader(
					is));
			StringBuffer sb = new StringBuffer();
			String rLine = null;
			String tmp_rLine = null;
			while ((rLine = bReader.readLine()) != null) {
				tmp_rLine = rLine;
				int str_len = tmp_rLine.length();
				if (str_len > 0) {
					sb.append("\n" + tmp_rLine);
					// pw.println(tmp_rLine);
					// pw.flush();
					if (deepUrls.get(strUrl) < webDepth)
						getUrlByString(tmp_rLine, strUrl);
				}
				tmp_rLine = null;
			}
			is.close();
			// pw.close();
			if (!this.searchCurrent) {
				searchKey(getContent(strUrl), this.getSearchKey());
			}
			if (this.saveUrl) {
				searchedUrl.add(strUrl);
			}
			// System.out.println("Get web successfully! " + strUrl);
			// addReport("Get web successfully! " + strUrl + "\n");
			addWebSuccessed();
		} catch (Exception e) {
			System.out.println("Get web failed!       " + strUrl);
			// addReport("Get web failed! " + strUrl + "\n");
			addWebFailed();
		}
	}

	public String getDomain() {
		String reg = "(?<=http\\://[a-zA-Z0-9]{0,100}[.]{0,1})[^.\\s]*?\\.(com|cn|net|org|biz|info|cc|tv)";
		Pattern p = Pattern.compile(reg, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(strHomePage);
		boolean blnp = m.find();
		if (blnp == true) {
			return m.group(0);
		}
		return null;
	}

	public void searchKey(String content, String key) {
		try {
			Pattern p = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
			Matcher m = p.matcher(content);
			boolean blnp = m.find();

			SpiderDAO dao = new SpiderDAO();
			while (blnp == true) {
				String searched = m.group(0);
				if (this.saveSearchKey && searched != null
						&& !searched.trim().replace(" ", "").equals("null")
						&& !searched.trim().replace(" ", "").equals("")) {
					dao.addStockNum(searched, "", "");
					searchedKey.add(searched);
				}
				blnp = m.find();
			}
			System.out.println("-----" + searchedKey.size());
		} catch (Exception e) {
		}
	}

	public void getUrlByString(String inputArgs, String strUrl) {
		String tmpStr = inputArgs;
		String regUrl = "(?<=(href=)[\"]?[\']?)[http://][^\\s\"\'\\?]*("
				+ myDomain + ")[^\\s\"\'>]*";
		Pattern p = Pattern.compile(regUrl, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(tmpStr);
		boolean blnp = m.find();
		// int i = 0;
		while (blnp == true) {
			if (!allUrls.containsKey(m.group(0))) {
				// System.out.println("Find a new url,depth:" +
				// (deepUrls.get(strUrl) + 1) + " " + m.group(0));
				// addReport("Find a new url,depth:" + (deepUrls.get(strUrl) +
				// 1) + " " + m.group(0) + "\n");
				arrUrls.add(m.group(0));
				arrUrl.add(m.group(0));
				allUrls.put(m.group(0), getIntWebIndex());
				deepUrls.put(m.group(0), (deepUrls.get(strUrl) + 1));
			}
			tmpStr = tmpStr.substring(m.end(), tmpStr.length());
			m = p.matcher(tmpStr);
			blnp = m.find();
		}
	}

	public static void main(String[] args) {

		String url = "http://www.ip38.com";// "http://hi.baidu.com/guoyan227/blog/item/b759012375f0a846925807df.html";//http://www.google.com.hk/search?q=tom&hl=zh-CN&safe=strict&start=10&sa=N
		// url = "http://www.google.com.hk/search?hl=zh-CN&q=javagoogle";
		// url = "http://www.baidu.com/s?bs=%C0%B5%CE%B0%C3%B7&f=8&wd=%CB%F9%D3%D0%BA%BA%D7%D6";
		url = "http://www.baidu.com/s?wd=fbote&n=2";
		String key = "(\\d){1,3}\\.(\\d){1,3}\\.(\\d){1,3}\\.(\\d){1,3}";

		// url =
		// "http://hi.baidu.com/festsoft/blog/item/171feb8860eee492a5c27256.html";
		// key = "[\u4e00-\u9fa5]{0,1}";//[\u4e00-\u9fa5]{0,}
		GetWeb gw = new GetWeb(url, key);
		// gw.setSaveSearchKey(false);
		gw.setSearchCurrent(true);
		gw.getWebByHomePage();

		List list = gw.getSearchedKey();
		// SpiderDAO dao = new SpiderDAO();
		// dao.getStock();
		for (int i = 0; i < list.size(); i++) {
			// dao.addStockNum((String)list.get(i), "", "");
			System.out.println(list.get(i));
		}
		System.out.println("gsize:" + list.size());
		if (list.size() == 0) {
			list = gw.getSearchedUrl();
			System.out.println(list.size());
			for (int i = 0; i < list.size(); i++) {
				System.out.println(list.get(i));
			}
		}
	}

	class Processer implements Runnable {
		GetWeb gw;

		public Processer(GetWeb g) {
			this.gw = g;
		}

		public void run() {
			// Thread.sleep(5000);
			while (!arrUrls.isEmpty()) {
				String tmp = getAUrl();
				getWebByUrl(tmp, charset, allUrls.get(tmp) + "");
			}
		}
	}
}